84 possible refactor - #92
Conversation
…rse which is high-level and has more potential
…unners loaded from yml files
| @@ -1,4 +1,4 @@ | |||
| runner_type: ollama | |||
| model: | |||
| models: | |||
There was a problem hiding this comment.
enable specifying the parameters under here
There was a problem hiding this comment.
combine this with runnery.yml under 'model_params' key and then pass that whole sub-dictionary to the model
| for task_dict in task_list: | ||
| tasks.append(Task.from_dict(task_dict,source_path=benchmark_path)) | ||
|
|
||
| runners = Bench.load_runners(benchmark_path) |
There was a problem hiding this comment.
call the from_file on benchlist instead and in the bench constructor
…folder of log folder not inside benchmark folder
brownsarahm
left a comment
brownsarahm
left a comment
There was a problem hiding this comment.
minor chagnes + please check docs for necessary updates
| The path to the benchmark folder | ||
| ''' | ||
| current_dir = os.getcwd() | ||
| os.chdir(benchmark_path) |
There was a problem hiding this comment.
add a comment why thi sis being done this way
| os.system("git branch -m main") | ||
| except: | ||
| print("git might not be initialized in your system. Please run \"git init . \" when setup") | ||
| # Get python gitignore template and create .gitignore |
There was a problem hiding this comment.
change "python" here to package
|
|
||
|
|
||
| def run(self, runner=BenchRunner(), log_dir=None, score=False): | ||
| def run(self, runner=BenchRunner(), log_dir=None): |
There was a problem hiding this comment.
is scoring no longer possible while running? I know we were removing from the file, but why not allow it?
| import subprocess | ||
| from datetime import datetime |
There was a problem hiding this comment.
all imports at the top of the file
| out = subprocess.run( | ||
| "git diff", | ||
| shell=True, | ||
| capture_output=True, | ||
| text=True) | ||
| if out.stdout.strip() != '': | ||
| # Get current date and time | ||
| string_timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") | ||
|
|
||
| subprocess.run( | ||
| f"git add -A", | ||
| shell=True, | ||
| stdout=subprocess.DEVNULL, | ||
| stderr=subprocess.DEVNULL) | ||
| subprocess.run( | ||
| f"git commit -m \"Bench run commit: {string_timestamp}\"", | ||
| shell=True, | ||
| stdout=subprocess.DEVNULL, | ||
| stderr=subprocess.DEVNULL) | ||
|
|
||
| commit = subprocess.run( | ||
| "git log -1 --oneline", | ||
| shell=True, | ||
| capture_output=True, | ||
| text=True).stdout.strip() | ||
| commit_hash, commit_message = commit.split(' ', 1) |
There was a problem hiding this comment.
what is all of this for? add comments
2 participants
Uh oh!
There was an error while loading. Please reload this page.